function test()

    clc
    close all
    bands = 224
    %%loadAllData(bands)  %% Step1：把原始数据导出为mat文件

    %% 7月26日的数据
    if exist('K:\', 'dir') == 7
        src_root_path = 'F:\12份土壤\sun';
    else
        src_root_path = 'D:\soil';
        src_root_path = 'D:\hou\soil';
% %         src_root_path = 'E:\Code\Matlab\sun';
    end

    datapath = [src_root_path, '\data'];%% 反射率提取后的 Mat数据文件
    dataInputPath = [src_root_path, '\input'];%% 原始光谱反射率TXT文件
    fileSavePath = [src_root_path, '\output'];%% R2，RMSE结果输出文件

    dataSrcFile = [dataInputPath, '\orgin-NPK-2024-10-24.xlsx'];
    dataExcelFile = [src_root_path, '\合并之后的sheets.xlsx'];
    bandFile = 'F:\12份土壤\sun\thesis\bands.txt';

    args.dataSrcFile = dataSrcFile;

    args.bands = bands;
    args.removeHead = 0;%去掉前10
    args.removeHead = 6;%去掉前10-5
    args.removeTail = 0;
    args.removeTail = 8;%%去掉后10-20

% %     args.dataSrcInput = dataInputPath;%%Step 1
    args.dataInputPath = dataInputPath;%%Step 1
    args.fileSavePath = fileSavePath;
    
    args.dataExcelFile = dataExcelFile;
    args.bandFile = bandFile;
 
    datetime
    args.step = DoSteps();%%设置step
    DoLoadAllData(datapath, args)
    
    % % % % % % % % % % % % % % % % % % % % % % % % % %       
    disp('All done');
    datetime
end

function DoLoadAllData(datapath, args)
    step1 = 1;      step2 = 2;     step3 = 4;       step4 = 8;
    step5 = 16;     step6 = 32;    step7 = 128;     step8 = 256;
    step9 = 512;    step10 = 1024; step11 = 2048;
    step12 = step11 * 2;    step13 = step12 * 2;    step14 = step13 * 2;
    step15 = step14 * 2;    step16 = step15 * 2;    step17 = step16 * 2;
    step18 = step17 * 2;    step19 = step18 * 2;    step20 = step19 * 2;
    step21 = step20 * 2;    step22 = step21 * 2;    step23 = step22 * 2;
    step24 = step23 * 2;    step25 = step24 * 2;    step26 = step25 * 2;
    step27 = step26 * 2;    step28 = step27 * 2;    step29 = step28 * 2;
    step30 = step29 * 2;    step31 = step30 * 2;    step32 = step31 * 2;
    step33 = step32 * 2;    step34 = step33 * 2;    step35 = step34 * 2;

    step = args.step; %%把step赋值平移到上一层，下面的统一找一个函数
    
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %     
    bMergeOne = true;
    bMergeOne = false;%%是否正反样本合并成一个数据

    nUseEachRemove = 0;%%每个元素采用统一测试集，测数据不删减2024-11-25，可用作在step31，找出原始数据集中的异常点
%     nUseEachRemove = 1;%%每个元素采用不同的测试集，后面不再采用
    nUseEachRemove = 2;%%每个元素采用统一测试集，进行相同元素删减，用作训练与测试
% 
    args.nUseEachRemove = nUseEachRemove;%%采用不同元素的删除
    args.nRemove = 11;%%测试集删掉前11个点0.295
    args.nRemove = 13;%%测试集删掉前13个点0.303
%     args.nRemove = 11;%%测试集删掉前11个点0.305, 联动删除异常点
%     args.nRemove = 10;%%测试集删掉前10*3个点0.298, 联动删除异常点


    delpos = 1:20;%% 老的全局删除（没有联动，共56个不重复），删完之后就是（训练集+测试集）
% %     delpos = 1:19;%%新的联动全局删除19*3

    bWholeTrain = true;%%2024-11-25, 训练集包含测试集
    bWholeTrain = false;%%训练集与测试集分离

    bCutAllAbnormal = true;%%删除整体数据的异常点，删完了再分训练集与测试集
% %     bCutAllAbnormal = false;%%用作找全局（整体）异常点的时候操作，不删除

    %%每次变动labmda，都要重新运行step6，重新得到pos-max-point-decrease文件
    %         lambda = 0.25  %%每个元素单独删15个
    %         lambda = 0.235%%每个元素单独删10个，共30个
    lambda = 0.295%%每个元素删10个，共30个
    lambda = 0.303%%2025-01-31
    %         lambda = 0.305%%2025-07-12  联动删整体，再联动删测试集
    % %         lambda = 0.298%%2025-02-20 ，删11*3个最新联动删整体，再联动删测试集

    args.lambda = lambda;


    rng(20241024)%%这个不能变，影响数据集的划分

    windowSizeMA = 7; %% for MA
    windowSizeFD = 3; %% for 一阶导数

% % % % 原来的stpe位置

    bands = args.bands;
    dataSrcFile = args.dataSrcFile;
    dataInputPath = args.dataInputPath;
    fileSavePath = args.fileSavePath;
    fileMatPath = datapath;
% % % % % % % % % % % % % % % % % % % % % % % % % % %     
    if bitand(step, step1)
        disp('Step 1. ...............');
     
        ret = Helper.loadExcelAllData();        
    end
% % % % % % % % % % % % % % % % % % % % % % % % % % % %     
     if bitand(step, step2)
        disp('Step 2. ...............');
% %         loadAllData(bands)
        [y_data, x_data, flag, seq, index] = loadExcelData(dataSrcFile);
        corrcoef(y_data) %%y_data:406*3, x_data: 403*210
% % %         plotmatrix(y_true)

% % %         Helper.plot12Curve(xx, '光谱曲线');
% %          y_true = [];
% %          x_all = [];
         %%只留=1,要不这个保留，要不下面保留一个
% % %          [y_true, x_all, flag2, seq2, index2] = removeData(y_data, x_data, flag, seq, index);

        %%只留flag=1,  可选在对测量的样本合并1个
        [y_true, x_proc, flag2, seq2, index2] = Helper.ProcessOriginalData2(y_data, ...
                                 x_data, flag, seq, index, bMergeOne);
%         Helper.DrawHistgram(y_true)


        %% 增加对flag=2进行合并处理
        [yy2, xx2] = Helper.ProcessOriginalData(y_data, x_data, flag, seq, index);
% % %         Helper.DrawHistgram(yy2)
        y_true = [y_true; yy2];
        x_proc = [x_proc; xx2];
        Helper.DrawHistgram(y_true)

        %%删除整体数据的异常点
        if bCutAllAbnormal == true
            fileSaveMaxPos = fullfile(dataInputPath, 'pos-max-point-decrease-all.txt');
            savePos = load(fileSaveMaxPos);
            
            savePos = savePos(delpos, 1:3);
            savePos = unique(savePos);
            [y_true, x_proc] = Helper.removePoints(savePos, y_true, x_proc);
        end

        size(y_true)

% % % % % %         Helper.DrawHistgram(y_true)
% % % % % %         ins = y_true(:, 3) > 100;%%筛掉小于100的数据
% % % % % %         y_true = y_true(ins, :);
% % % % % %         x_proc = x_proc(ins, :);
% % % % % %         Helper.DrawHistgram(y_true)

% % % % % % %         ins = y_true(:, 2) > 60 | y_true(:, 2) <40 ;%%
% % % % % % %         y_true = y_true(ins, :);
% % % % % % %         x_proc = x_proc(ins, :);
% % % % % % %         Helper.DrawHistgram(y_true)
% % % %


        cv = cvpartition(size(y_true, 1), 'HoldOut', lambda); % 创建分区对象

        if bWholeTrain == false
            trainIdx = cv.training; % 获取训练集索引
            testIdx  = cv.test; % 获取测试集索引
        else
            trainIdx = true(size(cv.training));
            testIdx  = cv.test; % 获取测试集索引
        end
        y_train  = y_true(trainIdx, :);
        y_test   = y_true(testIdx, :);
        x_train  = x_proc(trainIdx, :);
        x_test   = x_proc(testIdx, :);


% %         %%删除误差较大的数
%         removeIns1 = [197,228,227,23,115,113,240,149,166];
%         removeIns1 = [195,226,225,23,113,87,148,190,238];
% %         removeIns1 = [196,225,226,112,24,114,165,210,238];%%HolodOut=0.218, 没有删100的数据
% %         [y_train, x_train] = Helper.removePoints(removeIns1, y_train, x_train);
% % %
%         removeIns2 = [64,53,26,24,46,6,56,65,61];
% %         removeIns2 = [64,53,24,21,46,65,67,56];
% %           removeIns2 = [63,52,24,22,45,4,36,59,64];%%HolodOut=0.218, 没有删100的数据
% %         [y_test, x_test] = Helper.removePoints(removeIns2, y_test, x_test);


        %%%2024-11-25 每个元素采用不同的删除，得到新的测试集
        if nUseEachRemove == 1
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            [y_test, x_test, inns] = WaveHelper.GetInsFromPos(y_test, x_test, args);
            %%pos 做不到，不同pos
        elseif nUseEachRemove == 2
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            [y_test, x_test, pos] = Helper.removeSamePoints(args, y_test, x_test);

%             onePos = find(testIdx==1);
%             delpos = onePos(pos);%%获得真实的位置
%             testIdx(delpos) = false;%%没用到

            %%新调整的数据
            x_proc = [x_train; x_test];
            trainIdx = false(size(x_proc, 1), 1);
            testIdx  = false(size(x_proc, 1), 1);
            trainIdx(1: size(x_train, 1)) = true;
            testIdx(size(x_train, 1) + 1 : end) = true;

            y_true = [y_train; y_test];
% %             Helper.DrawHistgram(y_true)
% %             title('新调整的数据');
        end

        disp([size(y_train), size(y_test)])
        x_data = x_proc;
        y_data = y_true;
        save('y_train.txt', '-ascii', 'y_train');
        save('y_test.txt', '-ascii', 'y_test');
        save('x_train.txt', '-ascii', 'x_train');
        save('x_test.txt', '-ascii', 'x_test');
        
        
        %%保存数据用作论文，已保存2025-06-20
% % % % %         save(fullfile(fileSavePath, 'proc_data.mat'), 'y_true', 'x_proc');
% % % % %         Helper.StatConcentration(y_true);%%用作统计y
%         return;
    end

    if bitand(step, step3)
        disp('Step 3. ...............');

% %         Helper.showCorrelationEachBand(x_data, y_data);%相关性
     end
     % % % % % % % 预处理 % % % % % % % % % % % % % % % % % % % % % % % % % % %
     % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
     % % % 注意预处理x_data与y_data不用转置，但模型需要转置
     %%%% conitnumm removal
     if bitand(step, step9)
        disp('Step 9. Conitnumm Removal...............');
        bShow = false;
%         bShow = true;
        x_data = PreHelper.DoCR(x_proc', bShow);%%x_data, 204*24

        x_data = x_data';
        x_train = x_data(trainIdx, :);
        x_test = x_data(testIdx, :);%nUseEachRemove=0 与2的时候不用处理

        if nUseEachRemove == 1
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            y_test = y_true(testIdx, :);
            [y_test, x_test] = WaveHelper.GetInsFromPos(y_test, x_test, args);
        end
        figure
        plot(x_data')
     end

     %%%%%%%进行SG滤波
     if bitand(step, step10)
        disp('Step 10. SG滤波...............');

        x_data = PreHelper.DoSG(x_proc');%%x_data, 204*24

        x_data = x_data';
        x_train = x_data(trainIdx, :);
        x_test = x_data(testIdx, :);

        if nUseEachRemove == 1
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            y_test = y_true(testIdx, :);
            [y_test, x_test] = WaveHelper.GetInsFromPos(y_test, x_test, args);
        end
     end
     %%%%%%%进行MSC预处理
     if bitand(step, step11)
        disp('Step 11. MSC预处理...............');
        x_data = PreHelper.DoMSC(x_proc');%%x_data, 204*24

        x_data = x_data';
        x_train = x_data(trainIdx, :);
        x_test = x_data(testIdx, :);

        if nUseEachRemove == 1
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            y_test = y_true(testIdx, :);
            [y_test, x_test] = WaveHelper.GetInsFromPos(y_test, x_test, args);
        end
        
        FileSave = fullfile(fileSavePath, 'aaa.txt');
        save(FileSave, '-ascii', '-tabs', 'x_train');
% %         figure
% %         plot(x_data')
     end

     %%%%%%%进行SNV预处理
     if bitand(step, step12)
        disp('Step 12. SNV预处理...............');
        x_data = PreHelper.DoSNV(x_proc');%%x_data, 204*24

        x_data = x_data';
        x_train = x_data(trainIdx, :);
        x_test = x_data(testIdx, :);

        if nUseEachRemove == 1
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            y_test = y_true(testIdx, :);
            [y_test, x_test] = WaveHelper.GetInsFromPos(y_test, x_test, args);
        end
     end

     %%%%%%%进行MA滤波
     if bitand(step, step13)
        disp('Step 13. MA滤波...............');
        x_data = PreHelper.DoMA(x_proc', windowSizeMA);%%x_data, 204*24

        x_data = x_data';
        x_train = x_data(trainIdx, :);
        x_test = x_data(testIdx, :);

        if nUseEachRemove == 1
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            y_test = y_true(testIdx, :);
            [y_test, x_test] = WaveHelper.GetInsFromPos(y_test, x_test, args);
        end
     end

     %%%%%%%进行1阶导数滤波
     if bitand(step, step14)
        disp('Step 14. 1阶导数滤波...............');
        bShow = true;
        bShow = false;
        x_data = PreHelper.DoMA(x_proc', windowSizeFD);%先做三点平滑
        x_data = PreHelper.DoDiff(x_data, 1, bShow);%%x_data, 204*24
%         x_data = PreHelper.DoDiff(x_proc', 1, bShow);%%x_data, 204*24

        x_data = x_data';
        x_train = x_data(trainIdx, :);
        x_test = x_data(testIdx, :);

        if nUseEachRemove == 1
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            y_test = y_true(testIdx, :);
            [y_test, x_test] = WaveHelper.GetInsFromPos(y_test, x_test, args);
        end
     end

     %%%%%%%进行2阶导数滤波
     if bitand(step, step15)
        disp('Step 15. 2阶导数滤波...............');
        bShow = true
        bShow = false

        x_data = PreHelper.DoMA(x_proc', windowSizeFD);%先做三点平滑
        x_data = PreHelper.DoDiff(x_data, 2, bShow);
%         x_data = PreHelper.DoDiff(x_proc', 2, bShow);

        x_data = x_data';
        x_train = x_data(trainIdx, :);
        x_test = x_data(testIdx, :);

        if nUseEachRemove == 1
            args.fileSaveMaxPos = fullfile(dataInputPath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
            y_test = y_true(testIdx, :);
            [y_test, x_test] = WaveHelper.GetInsFromPos(y_test, x_test, args);
        end
     end


     % % % % % 特征/波段选择 % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
     % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
      %%%%%%%进行SPA预处理
     if bitand(step, step16)
        disp('Step 16. SPA处理...............');
        bShow = false;
        bShow = true;
        ncomp = 20
        ins = 1:3; %%表示3个元素
        kopt = FeaSelect.DoSPAEx(x_data, y_data, ncomp, ncomp, ins);

        args.metals = ins;
        args.wave = ncomp;
        [x_train, x_test] = FeaSelect.getXdataFromKopt(kopt, x_train, x_test, args);

     end

     %%%%%%%进行fsrftest滤波   Univariate Feature Ranking Using F-Tests
     if bitand(step, step17)
        disp('Step 17. fsrftest...............')
        bShow = false
        ncomp = 20
        kopt = FeaSelect.DoFsrfTest(x_data, y_data, ncomp, bShow);%%没有

        args.metals = 1:3;
        args.wave = ncomp;
        [x_train, x_test] = FeaSelect.getXdataFromKopt(kopt, x_train, x_test, args);


% % % % %         bShow = true
% % % % %         ncomp = 10
% % % % %         ret = MyModel.DoEachMLSRegressionGroup(x_data', y_data', fsss, bShow, ncomp)
% % % % %         fileSaveName = fullfile(fileSavePath, 'fsrftest-MLS-each.txt');
% % % % %         Helper.SaveStatsToFiles(fileSaveName, ret);
     end

     %%%%%%%进行fsrnca滤波
     if bitand(step, step18)
        disp('Step 18. fsrnca...............')
        %%效果不太好，结果不直观，有些comp只有1个，不用
        bShow = true;
        bShow = false;
        [ret, mdls] = FeaSelect.DoFsrNCA(x_data, y_data, bShow);%%
        fileSaveName = fullfile(fileSavePath, 'NCA-MLS-each.txt');
        Helper.SaveStatsToFiles(fileSaveName, ret);

% %         kopt = [1,88, 127, 199, 201];
% %         bShow = false;
% %         ncomp = 5;
% %         ret = Helper.DoEachMLSRegression(x_data', y_data', ncomp, kopt, bShow);
% %         fileSaveName = fullfile(fileSavePath, 'test-MLS-fsrnca.txt');
% %         Helper.SaveStatsToFiles(fileSaveName, ret);
     end


     %%%%%%%进行sequentialfs选择
     if bitand(step, step19)
        disp('Step 19. sequentialfs选择...............');
        %% 特征机制需要y，那就不能用
       bShow = false
        ncomp = 60;
%         kopt = FeaSelect.DoCarsNew(x_data, y_data, bShow, ncomp);
        kopt = FeaSelect.DoSFS(x_data, y_data);
        args.metals = 1:3;
        args.wave = ncomp;
        [x_train, x_test] = FeaSelect.getXdataFromKopt(kopt, x_train, x_test, args);

% %         fsss = FeaSelect.DoSFS(x_data', y_data');
% %         bShow = true
% %         ncomp = 10;
%         ret = MyModel.DoEachMLSRegressionGroup(x_data', y_data', kopt, bShow)
%         fileSaveName = fullfile(fileSavePath, 'carsnew-MLS-each.txt');

% %         Helper.SaveStatsToFiles(fileSaveName, ret);
     end

      %%%%%%%进行relieff选择
     if bitand(step, step20)
        disp('Step 20. relieff 选择...............');

        bShow = false
        ncomp = 20;
        kopt = FeaSelect.DoRelieff(x_data, y_data, bShow, ncomp);
%         bShow = true;
        args.metals = 1:3;
        args.wave = ncomp;
        [x_train, x_test] = FeaSelect.getXdataFromKopt(kopt, x_train, x_test, args);
     end

     %%%%%%%进行cars选择
     if bitand(step, step21)
        disp('Step 21. cars 选择...............');

        bShow = false
        ncomp = 20;
        kopt = FeaSelect.DoCars(x_data, y_data, bShow, ncomp);

        args.metals = 1:3;
        args.wave = 20;
        [x_train, x_test] = FeaSelect.getXdataFromKopt(kopt, x_train, x_test, args);



% %         bShow = true;
% %         ret = MyModel.DoEachMLSRegressionGroup(x_data', y_data', fsss, bShow, ncomp)
% %         fileSaveName = fullfile(fileSavePath, 'cars-MLS-.txt');
% %         Helper.SaveStatsToFiles(fileSaveName, ret);
     end

     %%%%%%%进行随机蛙跳选择
     if bitand(step, step22)
         %%不采用了，选择的波段比较集中
         %% 出来波段结果：182   183   146   181   184   174   179   186   173   185
        disp('Step 22. 随机蛙跳选择...............');
        addpath('F:\12份土壤\matlab\iRF')

        bShow = false
        bShow = true
        ncomp = 20;
        kopt = FeaSelect.DoRandomFrog(x_data, y_data, bShow, ncomp);

        args.metals = 1:3;
        args.wave = ncomp;%%12
        [x_train, x_test] = FeaSelect.getXdataFromKopt(kopt, x_train, x_test, args);

% %         bShow = true;
% %         ret = MyModel.DoEachMLSRegressionGroup(x_data', y_data', fsss, bShow, ncomp)
% %         fileSaveName = fullfile(fileSavePath, 'frog-MLS-.txt');
% %         Helper.SaveStatsToFiles(fileSaveName, ret);
     end
% % % % % % % % % % % % % % % % % % % % % % % % %
     if bitand(step, step23)
        disp('Step 23. 改为IRIV   LAR选择...............');
%         addpath('F:\12份土壤\matlab\lar')
%         addpath('F:\12份土壤\matlab\lars_lasso\lars')
        addpath('libPLS_1.98')
        bShow = false
        ncomp = 10;

        yy = y_data';
% %         F = iriv(x_data', yy(:, 4), 10, 6)
% %         save('iriv.mat', 'F');
        dataI = load('iriv.mat');
        F = dataI.F

%         fsss = FeaSelect.DoLAR(x_data', y_data', bShow, ncomp);
        bShow = true;
        x_proc = x_data';
        fsss = cell(6,1);
        fsss{1} = F.SelectedVariables;
        fsss{2} = F.SelectedVariables;
        fsss{3} = F.SelectedVariables;
        fsss{4} = F.SelectedVariables;
        fsss{5} = F.SelectedVariables;
        fsss{6} = F.SelectedVariables;
% %         xx = xx(:, F.SelectedVariables);
        ret = MyModel.DoEachMLSRegressionGroup(x_data', y_data', fsss, bShow, ncomp)
        fileSaveName = fullfile(fileSavePath, 'irir-MLS-.txt');
        Helper.SaveStatsToFiles(fileSaveName, ret);
     end

 % % % % % % % % % % % % % % % % % % % % % % % % %
     if bitand(step, step24)
        disp('Step 24. 随机森林选择...............');
        %%% 不采用了，和建模方法一样的话
        bShow = false
        ncomp = 10;
        fsss = FeaSelect.DoFeaRFTree(x_data', y_data', bShow, ncomp);
        bShow = true;
        ret = MyModel.DoEachMLSRegressionGroup(x_data', y_data', fsss, bShow, ncomp)
        fileSaveName = fullfile(fileSavePath, 'tree-MLS-each.txt');
        Helper.SaveStatsToFiles(fileSaveName, ret);
     end
   % % % % % % % % % % % % % % % % % % % % % % % % %
     if bitand(step, step25)
        disp('Step 25. UVE 选择...............');

        bShow = false
%         bShow = true
        ncomp = 20;
        kopt = FeaSelect.DoUVE(x_data, y_data, bShow, ncomp);
% % %         Helper.DrawSelectionOnMeanSpectral(x_data, kopt)

        args.metals = 1:3;
        args.wave = ncomp;
        [x_train, x_test] = FeaSelect.getXdataFromKopt(kopt, x_train, x_test, args);




% % % % % %         bShow = true;
% % % % % %
% % % % % %         ret = MyModel.DoEachMLSRegressionGroup(x_data', y_data', fsss, bShow, ncomp)
% % % % % %         fileSaveName = fullfile(fileSavePath, 'UVE-MLS-each.txt');
% % % % % %         Helper.SaveStatsToFiles(fileSaveName, ret);
     end
     % % % % % % % % % % % % % % %      % % % % % % % % % % % % % % % % % % % % % % %
     % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
     % % % % % 回归模型 % % % % % % % % % % % % % % % % % % % % % % % % % % %
     % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
     %%%%%%偏最小二乘回归
     if bitand(step, step4)
        disp('Step 4.偏最小二乘回归 ...............');

        args.bShow = false;
        args.ncomp = 8; %%7;%%回归系数，不是波段数，8最好 2025-02-02（用作测试集删除2025-07-13）

        [ret1, ret2, beta,~, MSE] = MyModel.DoPLSR(x_train, y_train, x_test, y_test, args)
% %         mse = MSE(2,1+arg.ncomp)%%只针对 cv取值    ，最后一个数

% % % % % % % %         fileSaveName = fullfile(fileSavePath, 'PLS-all.txt');
% % % % % % % %         Helper.SaveStatsToFiles(fileSaveName, ret1, ret2);
% % % % % % % %
% % % % % % % %         fileSaveName = fullfile(fileSavePath, 'yfit-PLS-all.txt');
% % % % % % % %         Helper.SaveValuesToFile(fileSaveName, ret1, ret2);

        bEach = false;
%         bEach = true;%% 显示逐个因变量的 PLS情况
        args.bEach = bEach;
        [ret1, ret2, mse] = MyModel.DoEachRegression(x_train, y_train, x_test, y_test, args) ;

        Helper.DrawResidua(ret1.y, ret1.yfit);
        Helper.DrawResidua(ret2.y, ret2.yfit, true);

        fileSaveName = fullfile(fileSavePath, 'PLS-each.txt');
        Helper.SaveStatsToFiles(fileSaveName, ret1, ret2);

        fileSaveName = fullfile(fileSavePath, 'yfit-PLS-each.txt');
        Helper.SaveValuesToFile(fileSaveName, ret1, ret2);

        fileSaveRetName = fullfile(fileSavePath, 'rets-y-fit.mat');
        save(fileSaveRetName, 'ret1', 'ret2');%%保存MAT文件
% % % % %         return;
% % % % % % % % %         zX = X - mean(X);
% % % % % % % % %         zY = y - mean(y);
% % % % %         [zX, muX, sigmaX] = zscore(X);%%正规化竟然效果不好
% % % % %         [zY, muY, sigmaY] = zscore(y);
% % % % % % % % %         muY = mean(y);
% % % % % % % % %         sigmaY = 1;
% % % % % 
% % % % %         Helper.PlotPLSRComps(zX, zY, nn);
% % % % %         beta = MyModel.DoPLSR(zX, zY, ncomp, bShow);
% % % % % 
% % % % %         aa = [ones(size(zX,1),1) zX] * beta;
% % % % %         y_hat = sigmaY .* aa + muY;
% % % % % 
% % % % %         Helper.PlotStemErrors(y, y_hat)
% % % % %         ret = Helper.CalcAllErrors(y, y_hat, bShow);
     end

     %%支持向量回归
     if bitand(step, step5)
        disp('Step 5. 支持向量回归...............');
% % % %         column = 4;
% % % %         y_data = repmat(y_true, 1, 2);
% % % %         x_data =  Helper.extractMatrix(xx, column);
        bShow = false;
        args.bShow = bShow;

%         X = X - mean(X, 1);
%         y = y - mean(y, 1);

%         type = 5;
%         type = 4;
%         type = 3;
%         type = 2;
%         type = 1;
%         ret = MyModel.DoEachSVR(X, y, bShow, type);
%         fileSaveName = fullfile(fileSavePath, ['SVR-', num2str(type), '.txt']);
%         Helper.SaveStatsToFiles(fileSaveName, ret);
        rng(20230214)

        for type = 2 :4  %type=1,5不要了太差了
            args.type = type;
            [ret1, ret2] = MyModel.DoEachSVR(x_train, y_train, x_test, y_test, args);
            fileSaveName = fullfile(fileSavePath, ['SVR-', num2str(type), '.txt']);
            Helper.SaveStatsToFiles(fileSaveName, ret1, ret2);

            fileSaveName = fullfile(fileSavePath, ['yfit-SVR-', num2str(type), '.txt']);
            Helper.SaveValuesToFile(fileSaveName, ret1, ret2);

            Helper.DrawResidua(y_train, ret1.yfit);
            Helper.DrawResidua(y_test, ret2.yfit, true);
%             Helper.plotPredict(y, yfit)
        end

        fileMerge = fullfile(fileSavePath, 'aaa.cmd');
        status = system(fileMerge);
        if status == 0
            disp('批处理合并文件成功执行\n');
        else
            fprintf('批处理文件执行失败，错误代码：%d\n', status);
        end
     end
     %% 决策树 ，随机森林
     if bitand(step, step6)
        disp('Step 6. 随机森林 ...............');
% %         column = 4;
% %         y_data = repmat(y_true, 1, 2);
% %         x_data =  Helper.extractMatrix(xx, column);
        bShow = false;
        args.bShow = bShow;
        rng(20241025)
% % %         kfold = cc.NumTestSets;
% % %         kfold = 0;  %%记得要在此设置


% % % % %          %%找出最优分量
% % % % %         ret = MyModel.DoEachRFwithBest(X, y, bShow);
% % % % %         fileSaveName = fullfile(fileSavePath, ['best_tree-1.txt']);
% % % % %         Helper.SaveStatsToFiles(fileSaveName, ret);
% % % % %         return

%         return
        %%决策树
% % % % %         ret = Helper.DoEachTree(X, y, bShow);
% % % % %         fileSaveName = fullfile(fileSavePath, ['fitrtree.txt']);
% % % % %         Helper.SaveStatsToFiles(fileSaveName, ret);
% % % % % % % %         return

        %%随机森林
        [ret1, ret2] = MyModel.DoEachRF(x_train, y_train, x_test, y_test, args);
%         ret = [ret1, ret2]

        bShowPoint = true;%%显示每个点的文本
%         bShowPoint = false;

        Helper.DrawResidua(y_train, ret1.yfit, false);
        Helper.DrawResidua(y_test, ret2.yfit, bShowPoint);

        fileSaveName = fullfile(fileSavePath, 'random_tree.txt');
        Helper.SaveStatsToFiles(fileSaveName, ret1, ret2);

        fileSaveName = fullfile(fileSavePath, 'yfit-random_tree.txt');
        Helper.SaveValuesToFile(fileSaveName, ret1, ret2);

% %         fileSaveName = fullfile(fileSavePath, 'diff-random_tree.txt');
% %         Helper.SaveSortDiffValue(fileSaveName, ret1, ret2);

        fileSaveRetName = fullfile(fileSavePath, 'rets-y-fit.mat');
        save(fileSaveRetName, 'ret1', 'ret2');%%保存MAT文件

% % % % %         [ret1, ret2] = MyModel.DoLinearModel(ret1.yfit, ret1.y, ret2.yfit, ret2.y, args)
% % % % %         Helper.DrawResidua(y_train, ret1.yfit, false);
% % % % %         Helper.DrawResidua(y_test, ret2.yfit, bShowPoint);
% % % % %
% % % % %         fileSaveName = fullfile(fileSavePath, 'random_tree.txt');
% % % % %         Helper.SaveStatsToFiles(fileSaveName, ret1, ret2);

% % %         fileSaveName = fullfile(fileSavePath, 'pos-max-point-decrease.txt');
% % %         args.fileSaveMaxPos = fullfile(fileSavePath, 'max-pos-diff.txt');
% % %         args.klen = 25;
% % %         [savePos, saveM] = Helper.FindMaxDiff(fileSaveName, ret2.y, ret2.yfit, args);%%测试集删点
% % %         savePos = savePos(1:2, :);
% % %         savePos = savePos(:)
% % %         [ytrue, ypred] = Helper.removePoints(savePos, ret2.y, ret2.yfit);
% % %         Helper.DrawResidua(ytrue, ypred, true);

% % %         OK，去掉训练集的最差的3个点
% % %         args.klen = 40;
% % %         [savePos, saveM] = Helper.FindMaxDiff(fileSaveName, ret1.y, ret1.yfit, args);%%训练集删点
% % %         savePos = savePos(1:3, :);
% % %         savePos = savePos(:)
% % %         [ytrue, ypred] = Helper.removePoints(savePos, ret1.y,ret1.yfit);
% % %         Helper.DrawResidua(ytrue, ypred, false);
     end
     %% LSBoost
     if bitand(step, step8)
        disp('Step 8. LSBoost ...............');
        bShow = false;
        args.bShow = bShow;

        rng(20241025)
%         rng(20250715)
        bShowPoint = true;%%显示每个点的文本

        [ret1, ret2] = MyModel.DoEachLSB(x_train, y_train, x_test, y_test, args);

        Helper.DrawResidua(y_train, ret1.yfit);
        Helper.DrawResidua(y_test, ret2.yfit, bShowPoint);

        fileSaveName = fullfile(fileSavePath, ['LSBoost.txt']);
        Helper.SaveStatsToFiles(fileSaveName, ret1, ret2);

        fileSaveName = fullfile(fileSavePath, 'yfit-LSBoost.txt');
        Helper.SaveValuesToFile(fileSaveName, ret1, ret2);

     end
    %      lasso太慢，没用，改为神经性网络
     if bitand(step, step7)
        disp('Step 7. 神经网络...............');

        bShow = false;
        bShowPoint = true;%%显示每个点的文本
        args.bShow = bShow;
        
        seed = 20241025;
        rng(seed)
        args.seed = seed;
        args.showANN = true;%%false, true 显示ANN训练窗口
        args.bUseNormal = true; %%是否进行每一特征归一化[-1, 1]
        
        [ret1, ret2] = MyModel.DoEachANN(x_train, y_train, x_test, y_test, args);

        Helper.DrawResidua(y_train, ret1.yfit);
        Helper.DrawResidua(y_test, ret2.yfit, bShowPoint);

        fileSaveName = fullfile(fileSavePath, 'ANN.txt');
        Helper.SaveStatsToFiles(fileSaveName, ret1, ret2);

        fileSaveName = fullfile(fileSavePath, 'yfit-ANN.txt');
        Helper.SaveValuesToFile(fileSaveName, ret1, ret2);


%         ret = Helper.DoEachLasso(X, y, bShow);
%         fileSaveName = fullfile(fileSavePath, 'lasso.txt');
%         Helper.SaveStatsToFiles(fileSaveName, ret);
     end

% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %
% % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % % %

    if bitand(step, step26)
        disp('Step 26. 特征与预处理的组合..针对随机森林.............');

        bShow = true;
        bShow = false;

        arg.bShow = bShow;
        arg.x_proc = x_proc;
        arg.y_true = y_true;
        arg.trainIdx = trainIdx;
        arg.testIdx = testIdx;
        arg.y_train = y_train;
        arg.y_test = y_test;
        arg.lambda = lambda;
        arg.nUseEachRemove = nUseEachRemove;
        arg.showANN = false;%%false, true 显示ANN训练窗口
        arg.bUseNormal = true; %%是否进行每一特征归一化[-1, 1]

        arg.wave = 20;
        arg.metals = 1:3;

        arg.pres = 6; %%4;预处理个数
        arg.feas = 6; %%6;特征选取个数
        arg.seed = 20241025;%%随机森立种子

        arg.withNoPre = 0; %%0-不需要预处理
        arg.withNoPre = 1; %%1-表示加上“没有预处理”开始

        arg.withNoFea = 1; %%-不做特征选择，只做预处理
        arg.withNoFea = 0; %%-做特征选择
        
        arg.modelType = 2;%%SVR 模型
%         arg.modelType = 3;%%RF 模型
%         arg.modelType = 4;%%LSBoost
%         arg.modelType = 5;%%ANN
        
        arg.type = 2;%%SVR的核类型
%         arg.type = 3;
%         arg.type = 4;
        
        ret = Helper.DoAllMetalsFeaAndPre(arg);

        fileSaveName = fullfile(fileSavePath, 'All-element-pre-fea.mat');
        save(fileSaveName, 'ret');
    end


    if bitand(step, step27)
        disp('Step 27. 特征选择结果保存...............');
        fileSaveName = fullfile(fileSavePath, 'All-element-pre-fea.mat');
        ret = load(fileSaveName, 'ret');
        ret = ret.ret;

        fileSave = fullfile(fileSavePath, 'All-element-pre-fea.txt');

        Helper.SaveResultsFromPreFea(ret, fileSave)
%         arg.fileSave = fileSave;
% %          fileSave = arg.fileSave;
%         for metal = 1 : size(methods, 1)
%              rrr = data{metal};
%              if isempty(rrr)
%                  continue
%              end
%
%              tmp = [rrr.rmse; rrr.R2; rrr.rsd; rrr.rpd; rrr.mape];
%              save(fileSave{metal}, '-ascii', '-tabs', 'tmp');
%          end
    end

    if bitand(step, step28)
        disp('Step 28. 获得波长后并显示选取波长...............');

        setting = [1, 3, 3;
                   1, 3, 6;
                   1, 3, 1];

        fileSave = cell(3, 1);
        fileSave{1} =  'waves-opt-metal-1.txt';
        fileSave{2} =  'waves-opt-metal-2.txt';
        fileSave{3} =  'waves-opt-metal-3.txt';

        for i = 1 : 3
            fileSave{i} = fullfile(fileSavePath, fileSave{i});
        end
        arg.fileSave = fileSave;
        arg.bShow = false;
        arg.wave = 20;

        kopt = zeros(3, arg.wave);
        for metal = 1 : size(setting, 1)
            preType   = setting(metal, 1);
            feaType   = setting(metal ,3);

            ret = Helper.DoOneMetalWithOneSetting(x_data, y_data, metal, preType, feaType, arg);
            kopt(metal, :) = ret;
            save(fileSave{metal}, '-ascii', '-tabs', 'ret');

            fsss{metal} = ret;
        end

        FeaSelect.plotBands(x_data', y_data', fsss, 10);
        FeaSelect.plotBandsEachMetal(x_data', y_data', fsss, 10, setting);

        fileSave = fullfile(fileSavePath, 'waves-opt-metals-all.txt');
        save(fileSave, '-ascii', '-tabs', 'kopt');
    end

% % % % % % % % % % % % % % % % % % % % % % % % % %
    if bitand(step, step29)
        disp('Step 29. 合并ret文件...............');

        file1 = fullfile(fileSavePath, 'All-metals-features1.mat');
        file2 = fullfile(fileSavePath, 'All-metals-features2.mat');
        fileSaveName = fullfile(fileSavePath, 'All-metals-features.mat');

        nCut = 5
        ret = Helper.MergeRetFiles(file1, file2, fileSaveName, nCut)
    end

% % % % % % % % % % % % % % % % % % % % % % % % % %
    if bitand(step, step30)
        disp('Step 30. 一个模型，多个元素多种预处理...............');

        bShow = false;

        arg.bShow = bShow;        
        arg.bEach = false;
        arg.ncomp = 7;%%7;   %%PLSR的回归系数    
        arg.preTypes =  6; %6;
        arg.wait = 8;
        arg.lambda = lambda;
        arg.nUseEachRemove = nUseEachRemove;

        arg.trainIdx = trainIdx;
        arg.testIdx = testIdx;
        arg.y_train = y_train;
        arg.y_test = y_test;
        arg.seed = 20241025;%%随机森立种子
%         arg.seed = 20250715;%%随机森立种子
        arg.fileSavePath = fileSavePath;
        arg.showANN = false;%%false, true 显示ANN训练窗口
        arg.bUseNormal = true; %%是否进行每一特征归一化[-1, 1]

        modelType = 1 %%PLSR
        modelType = 2 %%SRV
%         modelType = 3 %%RF
%         modelType = 4 %%LSBoost
%         modelType = 5 %%ANN

%         arg.type = 2;%%SVR的核类型
        arg.type = 3;
%         arg.type = 4;

        ret = Helper.DoOneModelWithAllSetting(x_data, modelType, arg)
        fileSave = fullfile(fileSavePath, ['all-preTypes-one-model-', num2str(modelType), '.mat']);
        save(fileSave, 'ret');
        
        dataCell = cell2mat(ret);
        
        fileSave = fullfile(fileSavePath, ['all-preTypes-one-model-', num2str(modelType), '.txt']);
        fileID = fopen(fileSave, 'w');           
        formatStr = repmat('%f\t%f\t%f\t\t', 1, arg.preTypes);
        formatStr = [formatStr, '\n'];
        fprintf(fileID, formatStr, dataCell');%要转置保存
        fclose(fileID);
        
% %         ret = load(fileSave, 'ret');
% %         ret = ret.ret;

% %         data = [];
% %         for preType = 1 : arg.preTypes
% %              rrr = ret{preType};
% %              ret1 = rrr{1};
% %              ret2 = rrr{2};
% %              tmp = [rrr.rmse; rrr.R2; rrr.rsd; rrr.rpd; rrr.mape];
% %              zzz = zeros(size(tmp, 1), 1);
% % 
% %              if preType == 1
% %                  data = tmp;
% %              else
% %                  data = [data, zzz, tmp];
% %              end
% %         end
% % 
% %         fileSave = fullfile(fileSavePath, ['all-preTypes-one-model-', num2str(modelType), '.txt']);
% %         save(fileSave, '-ascii', '-tabs', 'data');
    end

% % % % % % % % % % % % % % % % % % % % % % % % % %
% % % % % % % % % % % % 需要先运行一个模型，如随机森林得出一个拟合结果
    if bitand(step, step31)
        fileSaveRetName = fullfile(fileSavePath, 'rets-y-fit.mat');
        data = load(fileSaveRetName, 'ret1', 'ret2');
        ret1 = data.ret1;%一个训练集结果，一个测试集结果
        ret2 = data.ret2;
        args.bShow = false;

% %         训练集找异常点
% %         Helper.DrawResidua(ret2.y, ret2.yfit, true);

% % %         [ret1, ret2, beta] = MyModel.DoLinearModel(ret1.yfit, ret1.y, ret2.yfit, ret2.y, args)
% % % %         [ret1, ret2, beta] = MyModel.DoLinearModel2(ret1.yfit, ret1.y, ret2.yfit, ret2.y, args)
% % %         Helper.DrawResidua(y_train, ret1.yfit, false);
% % %         Helper.DrawResidua(y_test, ret2.yfit, true);
% % %
% % %         fileSaveName = fullfile(fileSavePath, 'random_tree-linear.txt');
% % %         Helper.SaveStatsToFiles(fileSaveName, ret1, ret2);

%         return;

        %%测试集找异常点
        Helper.DrawResidua(ret2.y, ret2.yfit, true);

        args.fileSaveMaxPos = fullfile(fileSavePath, ['pos-max-point-decrease-', num2str(lambda), '.txt']);
        args.fileSaveDiffPos = fullfile(fileSavePath, 'max-pos-diff.txt');
        args.fileSaveTopPos = fullfile(fileSavePath, 'save-pos-test.txt');
                
        args.klen = 13;%联动时候设置小一点
        args.delpos = 1:11;

% %         savePos = WaveHelper.FindAbnormalPoints(ret2.y, ret2.yfit, args)
        savePos = WaveHelper.FindAbnormalPoints2(ret2.y, ret2.yfit, args)%%联动的找异常点
        savePos = unique(savePos)
        [ytrue, ypred] = Helper.removePoints(savePos, ret2.y, ret2.yfit);
        Helper.DrawResidua(ytrue, ypred, true);

        ret = Helper.CalcAllErrors(ytrue, ypred, false);
        fileSaveName = fullfile(fileSavePath, 'random_tree-remove.txt');
        Helper.SaveStatsToFiles(fileSaveName, ret);

% % %         OK，去掉训练集的最差的3个点
% % %         Helper.DrawResidua(ret1.y, ret1.yfit, false);
% % %         args.klen = 40;
% % %         args.fileSaveTopPos = fullfile(fileSavePath, 'save-pos-train.txt');
% % %         args.delpos = 1:3;
% % %         savePos = WaveHelper.FindAbnormalPoints(ret1.y, ret1.yfit, args)
% % %
% % %         [ytrue, ypred] = Helper.removePoints(savePos, ret1.y,ret1.yfit);
% % %         Helper.DrawResidua(ytrue, ypred, false);
    end
    % % % % % % % % % % % % % % % % % % % % % % % % % %
    %%%找全部数据（训练集+测试集）的异常点
    if bitand(step, step32)
        disp('step 32: 找全体数据集的异常点')
        fileSaveRetName = fullfile(fileSavePath, 'rets-y-fit.mat');
        data = load(fileSaveRetName, 'ret1', 'ret2');
        ret1 = data.ret1;
        ret2 = data.ret2;
        args.bShow = false;
        Helper.DrawResidua(ret1.y, ret1.yfit, true);


        args.fileSaveMaxPos = fullfile(fileSavePath, 'pos-max-point-decrease-all.txt');
        args.fileSaveDiffPos = fullfile(fileSavePath, 'max-pos-diff-all.txt');
        args.fileSaveTopPos = fullfile(fileSavePath, 'save-pos-test-all.txt');
        
        args.klen = 60;%%不联动
        args.delpos = 1:20;
        args.klen = 30;%%联动
        args.delpos = 1:25;

        savePos = WaveHelper.FindAbnormalPoints2(ret1.y, ret1.yfit, args)%%联动删除异常点
% % %         savePos = WaveHelper.FindAbnormalPoints(ret1.y, ret1.yfit, args)
        savePos = unique(savePos)
        [ytrue, ypred] = Helper.removePoints(savePos, ret1.y, ret1.yfit);
        Helper.DrawResidua(ytrue, ypred, true);

        ret = Helper.CalcAllErrors(ytrue, ypred, false);
        fileSaveName = fullfile(fileSavePath, 'random_tree-remove.txt');
        Helper.SaveStatsToFiles(fileSaveName, ret);
    end

    
    % % % % % % % % % % % % % % % % % % % % % % % % % %
    

end

% % Step2:  加载化学检测数据，加载mat波谱数据
function [y_true, xx, flag, seq, index] = loadExcelData(dataSrcFile)

    data = readtable(dataSrcFile,'Sheet','Sheet2');

    y_true = table2array(data(:,4:6));
    xx = table2array(data(:, 7:end));
    index = table2cell(data(:, 3));
    seq = table2array(data(:, 2));
    flag = table2array(data(:, 1));
end